{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 299580,
     "status": "ok",
     "timestamp": 1545896934122,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "yZzTzvPz99wr",
    "outputId": "2453b1c8-77d7-447a-a7dd-5fee376a202e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 5683,
     "status": "ok",
     "timestamp": 1545896942098,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "U0HLPpXzAB_-",
    "outputId": "8b87b20f-b2cf-4e5a-84ea-e338c10f7d04"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pymysql\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/39/15045ae46f2a123019aa968dfcba0396c161c20f855f11dea6796bcaae95/PyMySQL-0.9.3-py2.py3-none-any.whl (47kB)\n",
      "\u001b[K    100% |████████████████████████████████| 51kB 2.0MB/s \n",
      "\u001b[?25hInstalling collected packages: pymysql\n",
      "Successfully installed pymysql-0.9.3\n"
     ]
    }
   ],
   "source": [
    "!pip install pymysql"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "w2ai9wqbAMDc"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "g7x4QH7s_zIi"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pymysql\n",
    "import pymysql.cursors\n",
    "from functools import reduce\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import uuid\n",
    "import datetime\n",
    "#from pyfm import pylibfm\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "np.set_printoptions(precision=3)\n",
    "np.set_printoptions(suppress=True)\n",
    "from sklearn.datasets import dump_svmlight_file\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import pickle as pkl\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 7214,
     "status": "ok",
     "timestamp": 1545899077849,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "OTcqErrS-8CM",
    "outputId": "ab02e1d7-5f42-46e6-ffbf-389e9f4f6743"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (0,1,2,4,5,6,7,8,9,10,11,12,13,14,15,16,17,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('/content/drive/My Drive/colabnotebook/comment_origin_data_1.csv',sep='\\t',encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Bhr9HwDh-4Az"
   },
   "outputs": [],
   "source": [
    "#df_data.to_csv('/content/drive/My Drive/colabnotebook/comment_origin_data.csv', sep='\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 4440,
     "status": "ok",
     "timestamp": 1545899077856,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "m10beqzqATwN",
    "outputId": "fcf5fa41-f4d0-46ec-d8f4-41f45c2f2ec9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(389689, 29)"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "vNr0udIKAVLb"
   },
   "outputs": [],
   "source": [
    "df = df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "O_2l-KdFB02s"
   },
   "outputs": [],
   "source": [
    "df = df.drop(['Unnamed: 0'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 2444,
     "status": "ok",
     "timestamp": 1545899079304,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "UQ5HcXt6DJL_",
    "outputId": "6738a160-ced4-4ae1-9622-3aaa5c6554ba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289261,)"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['ID'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 2004,
     "status": "ok",
     "timestamp": 1545899079305,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "uZzPuEBvFz0i",
    "outputId": "83b7b67f-0c90-45e8-f233-799be1dd4b11"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289261, 28)"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "DUeP-5_cCxSu"
   },
   "outputs": [],
   "source": [
    "df = df.drop_duplicates(['ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "v4onpWhiF7Gi"
   },
   "outputs": [],
   "source": [
    "#df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "0ToH1iCCGDhJ"
   },
   "outputs": [],
   "source": [
    "#df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "vSYjBAI8GgTT"
   },
   "outputs": [],
   "source": [
    "df = df.drop(['CONTENT'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "vdprvLaaGmxS"
   },
   "outputs": [],
   "source": [
    "#df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "3ccgxkjhGovi"
   },
   "outputs": [],
   "source": [
    "df = df.drop(['ADD_TIME_x', 'ADD_TIME_y'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "b6YsB75OG_-5"
   },
   "outputs": [],
   "source": [
    "#df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 797,
     "status": "ok",
     "timestamp": 1545899085212,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "1SgvsDKHHBrZ",
    "outputId": "71eb0d91-047c-44e7-b5b6-939bdd470478"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289228, 25)"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "GD8kI3lYHDQA"
   },
   "outputs": [],
   "source": [
    "#df.iloc[10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "BqU3nZsjKFWw"
   },
   "outputs": [],
   "source": [
    "#df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "fTzf7qWAKzPO"
   },
   "outputs": [],
   "source": [
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "WlwazXJnLMNb"
   },
   "outputs": [],
   "source": [
    "#df.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "VHB1eN0QLRoK"
   },
   "outputs": [],
   "source": [
    "#df.head(1).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "VZ0ZuxtCMEwg"
   },
   "outputs": [],
   "source": [
    "df_main = df.drop(['name', 'CREATOR','description','img','ID','NEWDATA','MOVIEID'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 715,
     "status": "ok",
     "timestamp": 1545899090331,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "344kM6qMMhiB",
    "outputId": "de170997-122a-4fdb-d9a4-2d4b20099ce1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289228, 18)"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "DJ-7LA0MMjC_"
   },
   "outputs": [],
   "source": [
    "#df_main.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "st0zEq-xMkem"
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "ET5d9z5gNQFk"
   },
   "outputs": [],
   "source": [
    "#df_main.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "kXNLScDCNRMN"
   },
   "outputs": [],
   "source": [
    "df_main = df_main.drop(['enable'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "YsrCB_BhNS_j"
   },
   "outputs": [],
   "source": [
    "#df_main.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "6Jiap_1PN8i9"
   },
   "outputs": [],
   "source": [
    "#pd.to_datetime(df_main['TIME'], format='%d%b%Y:%H:%M:%S.%f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 818,
     "status": "ok",
     "timestamp": 1545899094760,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "kBTk5h1TTyqw",
    "outputId": "c38cf2ee-06a2-4cbf-bfde-4caf7b7744e0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datetime.datetime.strptime(df['TIME'][0],'%Y-%m-%d %H:%M:%S').year - 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "toyHqFyKZ33o"
   },
   "outputs": [],
   "source": [
    "df_main = df_main.dropna(subset=['USERID','rcount']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "8jImDpqbhnjw"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "gcykyyCoX8ny"
   },
   "outputs": [],
   "source": [
    "index = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "i79kOPFqWrzA"
   },
   "outputs": [],
   "source": [
    "def process_time(t):\n",
    "  global index\n",
    "  index = index + 1\n",
    "  try:\n",
    "    return datetime.datetime.strptime(t,'%Y-%m-%d %H:%M:%S').year - 2000\n",
    "  except:\n",
    "    print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "dDOUWBmOS7_Q"
   },
   "outputs": [],
   "source": [
    "df_main['TIME_DIS'] = df_main['TIME'].apply(lambda x: process_time(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 832,
     "status": "ok",
     "timestamp": 1545899108303,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "Eg9V_KcPYXxV",
    "outputId": "421ac5ae-b371-4b91-835c-306b03d5a479"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289152, 18)"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.dropna(subset=['USERID']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "aGLuXt-tSuGK"
   },
   "outputs": [],
   "source": [
    "#df_main.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 999,
     "status": "ok",
     "timestamp": 1545899110406,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "s8Am9BKMaNiJ",
    "outputId": "f7779e55-5abc-4a00-d94a-b349c376725d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289152, 18)"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "wRbGpt9aaXhZ"
   },
   "outputs": [],
   "source": [
    "df_main = df_main.drop(['TIME'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 869,
     "status": "ok",
     "timestamp": 1545899112808,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "RBhHUfbgajjB",
    "outputId": "6002114a-b4e2-488c-c8df-2ed50b5a2d83"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289152, 17)"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 714,
     "status": "ok",
     "timestamp": 1545899113762,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "4oYzCImHgkRO",
    "outputId": "bab62ef2-d7db-4343-fe17-0c15d0c32433"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289152, 16)"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.drop(['USERID'], axis=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "GbCxHzhdam-0"
   },
   "outputs": [],
   "source": [
    "#df_main.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "619P1B4hiOM5"
   },
   "outputs": [],
   "source": [
    "def get_connection():\n",
    "    return pymysql.connect(host='rm-2zeqqm6994abi7b6dqo.mysql.rds.aliyuncs.com',\n",
    "                               user='noone',\n",
    "                               password='Huawei12#$',\n",
    "                               db='recsys',\n",
    "                               port=3306,\n",
    "                               charset ='utf8',\n",
    "                               use_unicode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "8gfGRVb0iPFU"
   },
   "outputs": [],
   "source": [
    "df_movie = pd.read_sql_query(\"select * from movie\", get_connection())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "fOWWo6qJhYQ-"
   },
   "outputs": [],
   "source": [
    "def get_dim_dict(df, dim_name):\n",
    "  type_list = list(map(lambda x:x.split('|') ,df[dim_name]))\n",
    "  type_list = [x for l in type_list for x in l]\n",
    "  def reduce_func(x, y):\n",
    "    for i in x:\n",
    "      if i[0] == y[0][0]:\n",
    "        x.remove(i)\n",
    "        x.append(((i[0],i[1] + 1)))\n",
    "        return x\n",
    "    x.append(y[0])\n",
    "    return x\n",
    "  l = filter(lambda x:x != None, map(lambda x:[(x, 1)], type_list))\n",
    "  type_zip = reduce(reduce_func, list(l))\n",
    "  #type_list = sorted(list(set(type_list)))\n",
    "  #type_zip = zip(list(range(len(type_list))), type_list)\n",
    "  #print(len(type_zip))\n",
    "  #print(type_zip)\n",
    "  type_dict = {}\n",
    "  for i in type_zip:\n",
    "    type_dict[i[0]] = i[1]\n",
    "  return type_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "JudeBkGWhcvA"
   },
   "outputs": [],
   "source": [
    "type_dict = get_dim_dict(df_movie, 'type')\n",
    "actors_dict = get_dim_dict(df_movie, 'actors')\n",
    "director_dict = get_dim_dict(df_movie, 'director')\n",
    "trait_dict = get_dim_dict(df_movie, 'trait')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "IfRfjyqDwvBn"
   },
   "outputs": [],
   "source": [
    "with open('/content/drive/My Drive/colabnotebook/actors_dict', 'wb') as f:\n",
    "  pkl.dump(actors_dict, f)\n",
    "  \n",
    "with open('/content/drive/My Drive/colabnotebook/director_dict', 'wb') as f:\n",
    "  pkl.dump(director_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "OILpoTkHjFu3"
   },
   "outputs": [],
   "source": [
    "train_y = df_main['RATING']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "oNhUWBhzjbww"
   },
   "outputs": [],
   "source": [
    "df_main = df_main.drop(['RATING'], axis=1)\n",
    "df_main = df_main.drop(['userid'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 514
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 34716,
     "status": "ok",
     "timestamp": 1545834565753,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "DE76Z9wRjlwI",
    "outputId": "eea113b8-671f-4a11-acb8-8a47a64b8a7b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>USERID</th>\n",
       "      <td>cf2349f9c01f9a5cd4050aebd30ab74f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieid</th>\n",
       "      <td>10533913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <td>剧情|奇幻|冒险|喜剧</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actors</th>\n",
       "      <td>艾米·波勒|菲利丝·史密斯|理查德·坎德|比尔·哈德尔|刘易斯·布莱克</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>region</th>\n",
       "      <td>美国</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>director</th>\n",
       "      <td>彼特·道格特|罗纳尔多·德尔·卡门</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trait</th>\n",
       "      <td>感人|经典|励志</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rat</th>\n",
       "      <td>8.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmax</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmin</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ravg</th>\n",
       "      <td>3.85714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rcount</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rsum</th>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rmedian</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TIME_DIS</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            0\n",
       "USERID       cf2349f9c01f9a5cd4050aebd30ab74f\n",
       "movieid                              10533913\n",
       "type                              剧情|奇幻|冒险|喜剧\n",
       "actors    艾米·波勒|菲利丝·史密斯|理查德·坎德|比尔·哈德尔|刘易斯·布莱克\n",
       "region                                     美国\n",
       "director                    彼特·道格特|罗纳尔多·德尔·卡门\n",
       "trait                                感人|经典|励志\n",
       "rat                                       8.7\n",
       "rmax                                        5\n",
       "rmin                                        2\n",
       "ravg                                  3.85714\n",
       "rcount                                      7\n",
       "rsum                                       27\n",
       "rmedian                                     4\n",
       "TIME_DIS                                   15"
      ]
     },
     "execution_count": 57,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.head(1).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "P96xHn-BjpqN"
   },
   "outputs": [],
   "source": [
    "#df_main = df_main.drop([65416,45287,65414]).reset_index(drop=True)\n",
    "#df_main = df_main.drop([73432]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 33256,
     "status": "ok",
     "timestamp": 1545834565756,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "lh49VTSoYNz8",
    "outputId": "d91f3baf-a81f-4c6f-e5a6-dea1b9d6c354"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((289152, 15), (289152,))"
      ]
     },
     "execution_count": 59,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_main.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "-ILzscdAYngB"
   },
   "outputs": [],
   "source": [
    "invalid_data_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "F6cbL4iQdJP3"
   },
   "outputs": [],
   "source": [
    "data_dict_list = []\n",
    "for i in df_main.index:\n",
    "  _dict = {}\n",
    "  is_invalid = False\n",
    "  #type\n",
    "  for s_type in df_main.iloc[i]['type'].split('|'):\n",
    "    _dict[s_type] = 1\n",
    "  #actors\n",
    "  for s_actor in df_main.iloc[i]['actors'].split('|'):\n",
    "    if not s_actor in actors_dict:\n",
    "      print('invalid data index is ' + str(i))\n",
    "      invalid_data_list.append(i)\n",
    "      is_invalid = True\n",
    "      break\n",
    "    if actors_dict[s_actor] < 2:\n",
    "      _dict['other_actor'] = 1\n",
    "    else:\n",
    "      _dict[s_actor] = 1\n",
    "  if is_invalid == True:\n",
    "    continue;\n",
    "  #regios\n",
    "  _dict[df_main.iloc[i]['region']] = 1\n",
    "  #userid ...\n",
    "  _dict[df_main.iloc[i]['USERID']] = 1\n",
    "  _dict[str(df_main.iloc[i]['movieid'])] = 1\n",
    "  _dict['rat'] = df_main.iloc[i]['rat']\n",
    "  _dict['rmax'] = df_main.iloc[i]['rmax']\n",
    "  _dict['rmin'] = df_main.iloc[i]['rmin']\n",
    "  _dict['ravg'] = df_main.iloc[i]['ravg']\n",
    "  _dict['rcount'] = df_main.iloc[i]['rcount']\n",
    "  _dict['rsum'] = df_main.iloc[i]['rsum']\n",
    "  _dict['rmedian'] = df_main.iloc[i]['rmedian']\n",
    "  _dict['TIME_DIS'] = df_main.iloc[i]['TIME_DIS']\n",
    "  #\n",
    "  #director\n",
    "  for s_director in df_main.iloc[i]['director'].split('|'):\n",
    "    if director_dict[s_director] < 2:\n",
    "      _dict['other_director'] = 1\n",
    "    else:\n",
    "      _dict[s_director] = 1\n",
    "  #trait\n",
    "  for s_trait in df_main.iloc[i]['trait'].split('|'):\n",
    "    _dict[s_trait] = 1\n",
    "  data_dict_list.append(_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 725025,
     "status": "ok",
     "timestamp": 1545835259823,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "_RQFKnueb-Qi",
    "outputId": "094cf788-415e-4a72-d33c-1bdfca3a442a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 62,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 721773,
     "status": "ok",
     "timestamp": 1545835259823,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "3rS-DCRP60zr",
    "outputId": "688d932f-6dc5-4046-e45a-c297afa2c653"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "289152"
      ]
     },
     "execution_count": 63,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_dict_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 721446,
     "status": "ok",
     "timestamp": 1545835259823,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "tHa8EM6ZkAE9",
    "outputId": "54055a6a-2b5c-4ece-8a67-f6c70944ba5d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(289152,)"
      ]
     },
     "execution_count": 64,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Yb55fy8wX52D"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "1PaWnzjai-t8"
   },
   "outputs": [],
   "source": [
    "v = DictVectorizer()\n",
    "train_X = v.fit_transform(data_dict_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 726436,
     "status": "ok",
     "timestamp": 1545835266829,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "7z7ipuvF7Vzf",
    "outputId": "f6caaede-e5e1-4ca9-b0a1-4b04c96132fd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DictVectorizer(dtype=<class 'numpy.float64'>, separator='=', sort=True,\n",
       "        sparse=True)"
      ]
     },
     "execution_count": 66,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 726187,
     "status": "ok",
     "timestamp": 1545835266829,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "V9kxKYjK-1GU",
    "outputId": "a0a1f148-10ed-4903-c163-0507cf834046"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((289152, 75619), (289152,))"
      ]
     },
     "execution_count": 67,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "IDCDztbk_NFX"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "888GKAJrhHKB"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "fkEuwADHNOh5"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "4MTZsHCS40gE"
   },
   "outputs": [],
   "source": [
    "#pd.DataFrame(data=train_X, columns=v.feature_names_).head(1).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "JSsslBN1VvVe"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "oa6_dwhpL4G9"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 722994,
     "status": "ok",
     "timestamp": 1545835266851,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "v2pvdEC3BY8s",
    "outputId": "a3375865-2803-418a-d043-f8c00aea2fb4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((289152, 75619), (289152,))"
      ]
     },
     "execution_count": 69,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "AjrjfDv-Oqpo"
   },
   "outputs": [],
   "source": [
    "# def fitpredict_libfm(trainX, trainY, testX, classification=True, rank=8, n_iter=100):\n",
    "#     encoder = OneHotEncoder(handle_unknown='ignore').fit(trainX)\n",
    "#     trainX = encoder.transform(trainX)\n",
    "#     testX = encoder.transform(testX)\n",
    "#     print(type(trainX))\n",
    "#     train_file = '/content/drive/My Drive/colabnotebook/libfm_train.txt'\n",
    "#     test_file = '/content/drive/My Drive/colabnotebook/libfm_test.txt'\n",
    "#     with open(train_file, 'w') as f:\n",
    "#         dump_svmlight_file(trainX, trainY, f=f)\n",
    "#     with open(test_file, 'w') as f:\n",
    "#         dump_svmlight_file(testX, numpy.zeros(testX.shape[0]), f=f)\n",
    "#     task = 'c' if classification else 'r'\n",
    "#     console_output = !$LIBFM_PATH -task $task -method mcmc -train $train_file -test $test_file -iter $n_iter -dim '1,1,$rank' -out output.libfm\n",
    "    \n",
    "#     libfm_pred = pandas.read_csv('output.libfm', header=None).values.flatten()\n",
    "#     return libfm_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "YHotZRAcOxbn"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "boVS6ZCOO7VG"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "6YdId39EPV2k"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "U6d7mjH4PbQk"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "gt12LAxGPqQf"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "nr7IaqR0Pe-9"
   },
   "outputs": [],
   "source": [
    "train_X_ = train_X[0:280000]\n",
    "train_y_ = train_y[:280000]\n",
    "test_X_ = train_X[280000:]\n",
    "test_y_ = train_y[280000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "riPiFlRXcp-B"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 717785,
     "status": "ok",
     "timestamp": 1545835267225,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "8ESWT1AqcfFj",
    "outputId": "b23bbf9d-b758-4a6f-b4a2-07302a898e97"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((280000, 75619), (280000,), (9152, 75619), (9152,))"
      ]
     },
     "execution_count": 72,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X_.shape, train_y_.shape, test_X_.shape, test_y_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 716746,
     "status": "ok",
     "timestamp": 1545835267225,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "1kmp4GRXccpc",
    "outputId": "075decf6-09fc-44b9-cf07-c4208fe59a16"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 73,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_X_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "x7-_98KUPFgV"
   },
   "outputs": [],
   "source": [
    "## save to libFM file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "y8bP6KxOdGJU"
   },
   "outputs": [],
   "source": [
    "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train.txt'\n",
    "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test.txt'\n",
    "\n",
    "# dump_svmlight_file(train_X_, train_y_, train_file)\n",
    "# dump_svmlight_file(test_X_, test_y_, test_file)\n",
    "  \n",
    "  \n",
    "#   with open(test_file, 'w') as f:\n",
    "#       dump_svmlight_file(testX, numpy.zeros(testX.shape[0]), f=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "nP19V9yx-Vei"
   },
   "outputs": [],
   "source": [
    "test_y_lr_ = test_y_.apply(lambda x: 1 if int(x) > 3 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "XfaJSyRSeEfY"
   },
   "outputs": [],
   "source": [
    "train_y_lr_ = train_y_.apply(lambda x: 1 if int(x) > 3 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 714468,
     "status": "ok",
     "timestamp": 1545835267237,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "AmpJ0MBPdnCF",
    "outputId": "7cc3da71-67b0-46c4-de76-ec5f816726c9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((280000,), (9152,))"
      ]
     },
     "execution_count": 78,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y_lr_.shape, test_y_lr_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "34UQeFEA1kAI"
   },
   "outputs": [],
   "source": [
    "## save to lr file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "y_JAmWaNeLI3"
   },
   "outputs": [],
   "source": [
    "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train_lr.txt'\n",
    "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test_lr.txt'\n",
    "\n",
    "# dump_svmlight_file(train_X_, train_y_lr_, train_file)\n",
    "# dump_svmlight_file(test_X_, test_y_lr_, test_file)\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "zOfQNIluhgfH"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Ylmx9m7EykfF"
   },
   "outputs": [],
   "source": [
    "scaler = preprocessing.MaxAbsScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "o2VTEnoiy0V1"
   },
   "outputs": [],
   "source": [
    "train_X_scaling = scaler.fit_transform(train_X_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "81FptTTOzHOX"
   },
   "outputs": [],
   "source": [
    "test_X_scaling = scaler.fit_transform(test_X_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "rVU--bGpznna"
   },
   "outputs": [],
   "source": [
    "# save to scaling lr\n",
    "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train_scaling_lr.txt'\n",
    "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test_scaling_lr.txt'\n",
    "\n",
    "# dump_svmlight_file(train_X_scaling, train_y_lr_, train_file)\n",
    "# dump_svmlight_file(test_X_scaling, test_y_lr_, test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "gqTIl8h-zadJ"
   },
   "outputs": [],
   "source": [
    "train_file = '/content/drive/My Drive/colabnotebook/recsys_train_scaling.txt'\n",
    "test_file = '/content/drive/My Drive/colabnotebook/recsys_test_scaling.txt'\n",
    "\n",
    "dump_svmlight_file(train_X_scaling, train_y_, train_file)\n",
    "dump_svmlight_file(test_X_scaling, test_y_, test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "0DjA8s3Iyjms"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "TljaqNf9yjxH"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "NzK57HhPyj7o"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "xeMotHGCeXrX"
   },
   "outputs": [],
   "source": [
    "with open('/content/drive/My Drive/colabnotebook/dict2vec', 'wb') as f:\n",
    "  pkl.dump(v, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 800,
     "status": "error",
     "timestamp": 1545834268842,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "UNrvITmuhA7l",
    "outputId": "cccfa561-e654-4c30-eaf8-4b351eeb1eae"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-aff70e193ae8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mv_from_pkl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/My Drive/colabnotebook/dict2vec'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m   \u001b[0mv_from_pkl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/My Drive/colabnotebook/dict2vec'"
     ]
    }
   ],
   "source": [
    "v_from_pkl = None\n",
    "with open('/content/drive/My Drive/colabnotebook/dict2vec', 'rb') as f:\n",
    "  v_from_pkl = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 168
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 814,
     "status": "error",
     "timestamp": 1545834105701,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "bDUJ2FpJhJ-k",
    "outputId": "16f218a4-be3a-4a66-8775-699b4e012720"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-571ecc4e83d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpredict_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mv_from_pkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'美国'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'v_from_pkl' is not defined"
     ]
    }
   ],
   "source": [
    "predict_x = v_from_pkl.transform({'美国':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 750,
     "status": "ok",
     "timestamp": 1545828079950,
     "user": {
      "displayName": "Cheng Gavin",
      "photoUrl": "",
      "userId": "16557339152332771497"
     },
     "user_tz": -480
    },
    "id": "x0O5o-mYh9o_",
    "outputId": "e0c87b57-1a4a-487b-ad4f-7b93fbe3c699"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 75619)"
      ]
     },
     "execution_count": 296,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "8ZKu3yBAiJXu"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "colab": {
   "collapsed_sections": [],
   "name": "5_prepare_libsvm_data.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
